import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import seaborn as sns

from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA

from scipy.cluster import hierarchy

# Set visualization style
sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = [10, 6]

# Define the files's for the 3 machine level CSV files
file_m01 = 'midterm_machine_01.csv'
file_m02 = 'midterm_machine_02.csv'
file_m03 = 'midterm_machine_03.csv'

# read in the CSV files and name them accordingly
m01_df = pd.read_csv(file_m01)
m02_df = pd.read_csv(file_m02)
m03_df = pd.read_csv(file_m03)

# Display the dimensions of each dataset
print("How many rows and columns are in each data set? Answers below: (rows, columns)\n")
print("- Machine 1 dataset shape:", m01_df.shape)
print("- Machine 2 dataset shape:", m02_df.shape)
print("- Machine 3 dataset shape:", m03_df.shape)

# Display names and data types for each column
print("\n\nWhat are the names and data types for each column?")
print("\n- Data types for Machine 1:\n")
print(m01_df.dtypes)
print("\n- Data types for Machine 2:\n")
print(m02_df.dtypes)
print("\n- Data types for Machine 3:\n")
print(m03_df.dtypes)

# Display number of unique values for each column
print("\n\nHow many unique values are there for each column?")
print("\n- Unique values in Machine 1:\n")
for col in m01_df.columns:
    print(f"{col}: {m01_df[col].nunique()} unique values")
print("\n- Unique values in Machine 2:\n")
for col in m02_df.columns:
    print(f"{col}: {m02_df[col].nunique()} unique values")
print("\n- Unique values in Machine 3:\n")
for col in m03_df.columns:
    print(f"{col}: {m03_df[col].nunique()} unique values")

# Check for missing values for each columns
print("\n\nHow many missing values are there for each column?")
print("\nMissing values in Machine 1:")
print(m01_df.isnull().sum())
print("\nMissing values in Machine 2:")
print(m02_df.isnull().sum())
print("\nMissing values in Machine 3:")
print(m03_df.isnull().sum())

How many rows and columns are in each data set? Answers below: (rows, columns)

- Machine 1 dataset shape: (5152, 7)
- Machine 2 dataset shape: (5119, 7)
- Machine 3 dataset shape: (4458, 7)


What are the names and data types for each column?

- Data types for Machine 1:

ID        object
Batch      int64
s_id       int64
x1       float64
x2       float64
x3       float64
x4       float64
dtype: object

- Data types for Machine 2:

ID        object
Batch      int64
s_id       int64
x1       float64
x2       float64
x3       float64
x4       float64
dtype: object

- Data types for Machine 3:

ID        object
Batch      int64
s_id       int64
x1       float64
x2       float64
x3       float64
x4       float64
dtype: object


How many unique values are there for each column?

- Unique values in Machine 1:

ID: 5152 unique values
Batch: 50 unique values
s_id: 149 unique values
x1: 5152 unique values
x2: 5152 unique values
x3: 5152 unique values
x4: 5152 unique values

- Unique values in Machine 2:

ID: 5119 unique values
Batch: 50 unique values
s_id: 133 unique values
x1: 5119 unique values
x2: 5119 unique values
x3: 5119 unique values
x4: 5119 unique values

- Unique values in Machine 3:

ID: 4458 unique values
Batch: 41 unique values
s_id: 141 unique values
x1: 4458 unique values
x2: 4458 unique values
x3: 4458 unique values
x4: 4458 unique values


How many missing values are there for each column?

Missing values in Machine 1:
ID       0
Batch    0
s_id     0
x1       0
x2       0
x3       0
x4       0
dtype: int64

Missing values in Machine 2:
ID       0
Batch    0
s_id     0
x1       0
x2       0
x3       0
x4       0
dtype: int64

Missing values in Machine 3:
ID       0
Batch    0
s_id     0
x1       0
x2       0
x3       0
x4       0
dtype: int64

# Question: Visually explore the MARGINAL behavior of the variables

# Use histograms and KDE plots for continous variables
fig, axes = plt.subplots(2, 2, figsize=(14, 10))
axes = axes.flatten()

for i, var in enumerate(["x1", "x2", "x3", "x4"]):
    sns.histplot(data=m01_df, x=var, kde=True, color="blue", alpha=0.5, ax=axes[i], label="Machine 1")
    sns.histplot(data=m02_df, x=var, kde=True, color="green", alpha=0.5, ax=axes[i], label="Machine 2")
    sns.histplot(data=m03_df, x=var, kde=True, color="red", alpha=0.5, ax=axes[i], label="Machine 3")
    axes[i].set_title(f"Distribution of {var} across machines")
    axes[i].legend()

plt.suptitle("Machine Operating Parameters Comparison", fontsize=16, y=1.02)

plt.tight_layout()
plt.show()

# Question: How many unique values for Batch are associated with each MACHINE?
print("\n- Unique Batch values per machine:\n")
print(f"\tMachine 1: {m01_df['Batch'].nunique()} batches")
print(f"\tMachine 2: {m02_df['Batch'].nunique()} batches")
print(f"\tMachine 3: {m03_df['Batch'].nunique()} batches")

# Question: How many cell phone cases are associated with each Batch value for each MACHINE?
# NOTE for the professor: Didn't print out batch counts dfs (mn_batch_counts), as it became very cluttered. Please, refere to the below plots
m1_batch_counts = m01_df['Batch'].value_counts().sort_index().reset_index()
m1_batch_counts.columns = ['Batch', 'Count']
m2_batch_counts = m02_df['Batch'].value_counts().sort_index().reset_index()
m2_batch_counts.columns = ['Batch', 'Count']
m3_batch_counts = m03_df['Batch'].value_counts().sort_index().reset_index()
m3_batch_counts.columns = ['Batch', 'Count']

# Visualize counts of cell phone cases per batch for each machine
fig, axes = plt.subplots(3, 1, figsize=(14, 15))

# Machine 1 with beautiful Blues palette - adding Batch as hue for color variation
sns.barplot(x='Batch', y='Count', hue='Batch', data=m1_batch_counts, ax=axes[0], 
            palette="Blues_d", legend=False)
axes[0].set_title("Cell phone cases per Batch - Machine 1")
axes[0].set_xlabel("Batch")
axes[0].set_ylabel("Count")
axes[0].tick_params(axis='x', rotation=90)

# Machine 2 with beautiful Greens palette
sns.barplot(x='Batch', y='Count', hue='Batch', data=m2_batch_counts, ax=axes[1], 
            palette="Greens_d", legend=False)
axes[1].set_title("Cell phone cases per Batch - Machine 2")
axes[1].set_xlabel("Batch")
axes[1].set_ylabel("Count")
axes[1].tick_params(axis='x', rotation=90)

# Machine 3 with beautiful Reds palette
sns.barplot(x='Batch', y='Count', hue='Batch', data=m3_batch_counts, ax=axes[2], 
            palette="Reds_d", legend=False)
axes[2].set_title("Cell phone cases per Batch - Machine 3")
axes[2].set_xlabel("Batch")
axes[2].set_ylabel("Count")
axes[2].tick_params(axis='x', rotation=90)

plt.suptitle("Production Volume Distribution by Batch and Machine", fontsize=16, y=1.0)
plt.tight_layout()
plt.show()

- Unique Batch values per machine:

	Machine 1: 50 batches
	Machine 2: 50 batches
	Machine 3: 41 batches

# Question: Do the summary statistics of the OPERATING VARIABLES x1 through x4 vary across the three MACHINES?
print("\nSummary statistics for operating variables in Machine 1:")
print(m01_df[["x1", "x2", "x3", "x4"]].describe())

print("\nSummary statistics for operating variables in Machine 2:")
print(m02_df[["x1", "x2", "x3", "x4"]].describe())

print("\nSummary statistics for operating variables in Machine 3:")
print(m03_df[["x1", "x2", "x3", "x4"]].describe())

# Create a combined DataFrame for visualization
combined_data = []
for i, (df, machine_name) in enumerate(zip([m01_df, m02_df, m03_df], ['Machine 1', 'Machine 2', 'Machine 3'])):
    for var in ["x1", "x2", "x3", "x4"]:
        temp_df = pd.DataFrame({
            'Machine': machine_name,
            'Variable': var,
            'Value': df[var]
        })
        combined_data.append(temp_df)

combined_df = pd.concat(combined_data)

# Visualize the differences in distributions between machines
fig, axes = plt.subplots(1, 4, figsize=(20, 5))
machine_colors = {"Machine 1": "blue", "Machine 2": "green", "Machine 3": "red"}

for i, var in enumerate(["x1", "x2", "x3", "x4"]):
    var_data = combined_df[combined_df['Variable'] == var]
    sns.boxplot(x='Machine', y='Value', data=var_data, ax=axes[i], 
                hue='Machine', palette=machine_colors, legend=False)
    axes[i].set_title(f"{var} Distribution by Machine")
    axes[i].set_xlabel("Machine")
    axes[i].set_ylabel(var)

plt.suptitle("Comparison of Operating Variables Across Machines", fontsize=16, y=1.05)
plt.tight_layout()
plt.show()

Summary statistics for operating variables in Machine 1:
                x1           x2           x3           x4
count  5152.000000  5152.000000  5152.000000  5152.000000
mean     51.998218   102.207011    23.947216    10.628436
std       3.151912    12.888171     2.269325     1.542291
min      44.275379    65.810729    19.095832     7.644190
25%      49.335776    96.610288    22.334048     9.479969
50%      52.536206   104.487824    23.855690    10.356002
75%      54.662490   110.609442    25.448356    11.318120
max      58.647186   130.228828    29.477213    14.612911

Summary statistics for operating variables in Machine 2:
                x1           x2           x3           x4
count  5119.000000  5119.000000  5119.000000  5119.000000
mean     51.989377   101.998905    23.982312    10.605275
std       3.095129    12.834348     2.275809     1.523140
min      44.896759    64.774007    19.232164     7.934064
25%      49.240567    96.325502    22.372944     9.492226
50%      52.610949   104.357308    23.894465    10.329911
75%      54.785418   110.672960    25.546656    11.286062
max      58.503441   131.009046    29.093229    14.400437

Summary statistics for operating variables in Machine 3:
                x1           x2           x3           x4
count  4458.000000  4458.000000  4458.000000  4458.000000
mean     51.857672   101.650057    23.927679    10.602371
std       3.022646    12.243012     2.035196     1.554708
min      45.690397    70.192150    19.865338     8.155668
25%      49.228020    95.381831    22.457249     9.475467
50%      52.598712   104.712727    23.900193    10.351781
75%      54.810528   110.050211    25.142293    11.301126
max      56.981029   126.332105    28.972747    14.182623

# Question: Do the summary statistics of the OPERATING VARIABLES x1 through x4 vary across the Batch values?
# Using boxplots to visualize summary statistics by Batch - with improved code and visualization

for machine_num, df, color in zip([1, 2, 3], [m01_df, m02_df, m03_df], ["#4285F4", "#34A853", "#EA4335"]):
    fig, axes = plt.subplots(1, 4, figsize=(20, 5))
    
    for i, var in enumerate(["x1", "x2", "x3", "x4"]):
        # Improved boxplot with cleaner technique
        ax = axes[i]
        
        # Create boxplot
        sns.boxplot(x='Batch', y=var, data=df, ax=ax, color=color, 
                   flierprops={'marker': 'o', 'markerfacecolor': 'white', 'markeredgecolor': color})
        
        # Add a subtle swarm plot to show actual data points (optional)
        if df['Batch'].nunique() < 15:  # Only add points if not too many batches
            sns.swarmplot(x='Batch', y=var, data=df, ax=ax, color='black', alpha=0.5, size=3)
        
        # Properly set title and labels
        ax.set_title(f"{var} by Batch - Machine {machine_num}", fontweight='bold')
        ax.set_xlabel("Batch")
        ax.set_ylabel(var, fontweight='bold')
        
        # Fix the tick labels warning by properly setting ticks
        # Get unique batch values
        batches = sorted(df['Batch'].unique())
        
        # Set x-ticks and labels correctly
        ax.set_xticks(range(len(batches)))
        ax.set_xticklabels(batches, rotation=90)
        
        # Add a subtle grid for better readability
        ax.grid(axis='y', linestyle='--', alpha=0.3)
        
        # Add batch value means as text annotations (optional for detailed analysis)
        means = df.groupby('Batch')[var].mean()
        for j, batch in enumerate(batches):
            if j % 2 == 0:  # Annotate every other batch to avoid crowding
                ax.annotate(f'{means[batch]:.1f}', 
                           (j, means[batch]), 
                           textcoords="offset points",
                           xytext=(0,10), 
                           ha='center', 
                           fontsize=8,
                           alpha=0.7)
    
    plt.suptitle(f"Operating Variable Variation by Batch - Machine {machine_num}", fontsize=16, y=1.02)
    plt.tight_layout()
    plt.show()

# Question: Do the relationships between the OPERATING VARIABLES x1 through x4 vary across the three MACHINES?
# Exploring relationships between variables using pairplots
for machine_num, df, color in zip([1, 2, 3], [m01_df, m02_df, m03_df], ["blue", "green", "red"]):
    sns.pairplot(df[["x1", "x2", "x3", "x4"]], diag_kind="kde", plot_kws={"alpha": 0.6})
    plt.suptitle(f"Operating Variable Relationships - Machine {machine_num}", y=1.02, fontsize=16)
    plt.show()

# Question: Do the relationships between the OPERATING VARIABLES x1 through x4 vary across the Batch values?
# Look at relationships between variables by Batch
for machine_num, df in zip([1, 2, 3], [m01_df, m02_df, m03_df]):
    g = sns.PairGrid(df, vars=["x1", "x2", "x3", "x4"], hue="Batch")
    g.map_diag(sns.histplot, alpha=0.6)
    g.map_offdiag(sns.scatterplot, alpha=0.6)
    g.add_legend(title="Batch", bbox_to_anchor=(1.05, 1), loc='upper left')
    plt.suptitle(f"Operating Variable Relationships by Batch - Machine {machine_num}", y=1.02, fontsize=16)
    plt.tight_layout()
    plt.show()

# CONCATENATE the 3 MACHINE data sets into a single DataFrame
# Add machine_id column to each DataFrame with the correct index value
m01_df['machine_id'] = 1
m02_df['machine_id'] = 2
m03_df['machine_id'] = 3

# Concatenate the dataframes
machine_df = pd.concat([m01_df, m02_df, m03_df], axis=0, ignore_index=True)

# Verify the concatenated dataframe
print("\nConcatenated machine DataFrame shape:", machine_df.shape)
print("\nDistribution of machine_id in concatenated DataFrame:")
print(machine_df['machine_id'].value_counts())
print("\nSample rows from concatenated DataFrame:")
machine_df.head()

Concatenated machine DataFrame shape: (14729, 8)

Distribution of machine_id in concatenated DataFrame:
machine_id
1    5152
2    5119
3    4458
Name: count, dtype: int64

Sample rows from concatenated DataFrame:

# Define the batch supplier file
batch_file = 'midterm_supplier.csv'

# Read in the batch supplier data set
batch_df = pd.read_csv(batch_file)

# Question: How many rows and columns are in the data?
print("- How many rows and columns are in the data?", batch_df.shape)

# Question: What are the names and data types for each column?
print("\n- What are the names and data types for each column?\n")
print(batch_df.dtypes)

# Question: How many unique values are there for each column?
print("\n- How many unique values are there for each column?")
for col in batch_df.columns:
    print(f"{col}: {batch_df[col].nunique()} unique values")

# Question: How many missing values are there for each column?
print("\n- How many missing values are there for each column?")
print(batch_df.isnull().sum())

- How many rows and columns are in the data? (50, 3)

- What are the names and data types for each column?

Batch         int64
Supplier     object
Density     float64
dtype: object

- How many unique values are there for each column?
Batch: 50 unique values
Supplier: 2 unique values
Density: 50 unique values

- How many missing values are there for each column?
Batch       0
Supplier    0
Density     0
dtype: int64

# Visualize the distribution of Density
plt.figure(figsize=(10, 6))
sns.histplot(data=batch_df, x="Density", kde=True, color="#4285F4")
plt.title("Distribution of Material Density", fontweight='bold')
plt.xlabel("Density")
plt.ylabel("Count")
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.suptitle("Material Density Analysis", fontsize=16, y=1.02, fontweight='bold')
plt.show()

# Show summary statistics for Density
print("\nSummary statistics for Density:")
print(batch_df['Density'].describe())

Summary statistics for Density:
count    50.000000
mean      9.817937
std       0.752225
min       8.703627
25%       9.294250
50%       9.679999
75%      10.156984
max      12.246089
Name: Density, dtype: float64

# Question: Do the summary statistics for Density depend on the Supplier?
# Visualize Density by Supplier using boxplot
plt.figure(figsize=(12, 7))
sns.boxplot(x='Supplier', y='Density', data=batch_df, palette=["#4285F4", "#34A853"], 
           hue='Supplier', legend=False)
# Add subtle data points
sns.stripplot(x='Supplier', y='Density', data=batch_df, 
             color='black', alpha=0.3, jitter=True, size=4)
plt.title("Density Distribution by Supplier", fontweight='bold', fontsize=14)
plt.xlabel("Supplier", fontweight='bold')
plt.ylabel("Density", fontweight='bold')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.suptitle("Material Density Variation by Supplier", fontsize=16, y=1.05, fontweight='bold')
plt.show()

# Calculate and display summary statistics by Supplier
print("\nSummary statistics for Density by Supplier:")
batch_df.groupby('Supplier')['Density'].describe()

Summary statistics for Density by Supplier:

# Question: Does the average Density depend on the Supplier?
# Calculate average Density by Supplier
avg_density_by_supplier = batch_df.groupby('Supplier')['Density'].mean().reset_index()

# Create barplot showing average Density by Supplier
plt.figure(figsize=(10, 6))
bars = sns.barplot(x='Supplier', y='Density', data=avg_density_by_supplier, 
                  palette=["#4285F4", "#34A853"], hue='Supplier', legend=False)

# Add value labels on the bars
for index, row in enumerate(avg_density_by_supplier['Density']):
    bars.text(index, row + 0.05, f'{row:.2f}', ha='center', fontweight='bold')

plt.title("Average Material Density by Supplier", fontweight='bold', fontsize=14)
plt.xlabel("Supplier", fontweight='bold')
plt.ylabel("Average Density", fontweight='bold')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.suptitle("Supplier Material Consistency Analysis", fontsize=16, y=1.05, fontweight='bold')
plt.show()

# Statistical test to check if the difference is significant
from scipy import stats
supplier_groups = batch_df.groupby('Supplier')['Density']
supplier_a = supplier_groups.get_group('A')
supplier_b = supplier_groups.get_group('B')
t_stat, p_value = stats.ttest_ind(supplier_a, supplier_b)
print(f"\nT-test for difference in mean Density between suppliers: p-value = {p_value:.4f}")

T-test for difference in mean Density between suppliers: p-value = 0.7827

# Question: How does Density relate to Batch for each Supplier?
plt.figure(figsize=(14, 7))
sns.lineplot(x='Batch', y='Density', hue='Supplier', data=batch_df, 
            palette={"A": "#4285F4", "B": "#34A853"}, 
            markers=True, dashes=False, markersize=8)
plt.title("Material Density Across Production Batches by Supplier", fontweight='bold', fontsize=14)
plt.xlabel("Batch", fontweight='bold')
plt.ylabel("Density", fontweight='bold')
plt.xticks(batch_df['Batch'].unique())
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend(title="Supplier")
plt.suptitle("Batch-level Material Quality Analysis", fontsize=16, y=1.05, fontweight='bold')
plt.show()

# merge the batch supplier data set with the (concatenated) machine data set
# Merge batch_df with machine_df
dfa = pd.merge(machine_df, batch_df, on='Batch', how='left')

# Check if any rows were lost in the merge
print(f"\nRows in machine_df: {len(machine_df)}")
print(f"Rows in dfa (after merge): {len(dfa)}")

# Verify the merged dataframe
print("\nMerged dataframe shape:", dfa.shape)
print("\nSample of merged data:")
dfa.head()

Rows in machine_df: 14729
Rows in dfa (after merge): 14729

Merged dataframe shape: (14729, 10)

Sample of merged data:

# Question: Explore if the summary statistics of the 4 OPERATING VARIABLES vary across Batch for each MACHINE given each Supplier
# Using facet grid to create multipanel visualization
operating_vars = ["x1", "x2", "x3", "x4"]

for var in operating_vars:
    g = sns.FacetGrid(dfa, col="machine_id", row="Supplier", height=4, aspect=1.5)
    
    # Define a custom function to create boxplots with proper order
    def boxplot_with_order(x, y, **kwargs):
        # Get the unique batches in sorted order
        batch_order = sorted(dfa['Batch'].unique())
        ax = plt.gca()
        sns.boxplot(x=x, y=y, order=batch_order, **kwargs)
        
        # Properly set ticks and then ticklabels
        ax.set_xticks(range(len(batch_order)))
        ax.set_xticklabels(batch_order, rotation=90)
    
    # Map the custom function
    g.map_dataframe(boxplot_with_order, "Batch", var)
    g.set_titles("Machine {col_name}, Supplier {row_name}")
    g.set_axis_labels("Batch", var)
    
    plt.suptitle(f"Distribution of {var} by Batch, Machine, and Supplier", fontsize=16, y=1.02)
    plt.tight_layout()
    plt.show()

# Question: Explore if the relationships between the 4 OPERATING VARIABLES vary across Supplier
# Create separate pairplots for each Supplier
for supplier in batch_df['Supplier'].unique():
    supplier_data = dfa[dfa['Supplier'] == supplier]
    sns.pairplot(supplier_data[["x1", "x2", "x3", "x4", "machine_id"]], 
                hue="machine_id", diag_kind="kde", 
                palette=["#4285F4", "#34A853", "#EA4335"])
    plt.suptitle(f"Operating Variable Relationships - Supplier {supplier}", 
                y=1.02, fontsize=16, fontweight='bold')
    plt.show()

# Define the test data file
test_file = 'midterm_test.csv'

# Read in the test data set
test_df = pd.read_csv(test_file)

# Question: How many rows and columns are in the data?
print("- How many rows and columns are in the data?", test_df.shape)

# Question: What are the names and data types for each column?
print("\n- What are the names and data types for each column?")
print(test_df.dtypes)

# Question: How many unique values are there for each column?
print("\n- How many unique values are there for each column?")
for col in test_df.columns:
    print(f"{col}: {test_df[col].nunique()} unique values")

# Question: How many missing values are there for each column?
print("\n- How many missing values are there for each column?")
print(test_df.isnull().sum())

- How many rows and columns are in the data? (1412, 3)

- What are the names and data types for each column?
ID               object
test_group_id    object
Result            int64
dtype: object

- How many unique values are there for each column?
ID: 1412 unique values
test_group_id: 141 unique values
Result: 2 unique values

- How many missing values are there for each column?
ID               0
test_group_id    0
Result           0
dtype: int64

# Visualize the distribution of Result
plt.figure(figsize=(10, 6))
result_counts = test_df['Result'].value_counts().sort_index()

# Create a DataFrame for proper seaborn plotting
result_df = pd.DataFrame({
    'Result': result_counts.index,
    'Count': result_counts.values
})

# Use the updated approach with hue parameter
bars = sns.barplot(x='Result', y='Count', hue='Result', 
                  data=result_df, palette="Blues_d", legend=False)

# Add value labels on the bars
for index, value in enumerate(result_counts.values):
    bars.text(index, value + 5, f'{value}', ha='center', fontweight='bold')

plt.title("Distribution of Drop Test Results", fontweight='bold')
plt.xlabel("Result")
plt.ylabel("Count")
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.suptitle("Drop Test Outcome Analysis", fontsize=16, y=1.02, fontweight='bold')
plt.show()

# Show summary statistics
print("\n- Summary of Result values:\n")
print(test_df['Result'].value_counts(normalize=True).sort_index() * 100)

- Summary of Result values:

Result
0    69.830028
1    30.169972
Name: proportion, dtype: float64

# Count the number of times each unique value of Result occurs for each test_group_id value
# Create a cross-tabulation of test_group_id vs Result
result_counts = pd.crosstab(test_df['test_group_id'], test_df['Result'])

# Add a total column
result_counts['Total'] = result_counts.sum(axis=1)

# Display the table
from IPython.display import display, HTML

# Convert to HTML and display
display(HTML(result_counts.to_html()))

dfa.head()

test_df.head()

# Merge test_df with dfa using the ID column which appears in both DataFrames
dfb = pd.merge(test_df, dfa, on='ID', how='inner')

# Question: How many rows remain using the DEFAULT joining procedure?
print("- How many rows remain using the DEFAULT joining procedure?")
print(f"\nOriginal test_df shape: {test_df.shape}")
print(f"Original dfa shape: {dfa.shape}")
print(f"Merged dfb shape: {dfb.shape}")
print(f"Number of rows after merging: {len(dfb)}")

# Calculate what percentage of test data was successfully merged
merge_percentage = (len(dfb) / len(test_df)) * 100
print(f"Percentage of test data rows retained in merge: {merge_percentage:.2f}%")

# Check for any ID values that didn't get matched in the merge
print("\nTest IDs not found in production data:")
missing_ids = set(test_df['ID']) - set(dfa['ID'])
print(missing_ids if missing_ids else "- All test IDs found in production data")

dfb.head()

- How many rows remain using the DEFAULT joining procedure?

Original test_df shape: (1412, 3)
Original dfa shape: (14729, 10)
Merged dfb shape: (1412, 12)
Number of rows after merging: 1412
Percentage of test data rows retained in merge: 100.00%

Test IDs not found in production data:
- All test IDs found in production data

# Count the number of times each unique value of Result occurs for each machine_id
result_by_machine = pd.crosstab(dfb['machine_id'], dfb['Result'])

# Add a total column for reference
result_by_machine['Total'] = result_by_machine.sum(axis=1)

# Display the table
print("Count of Results by Machine ID:")
result_by_machine

Count of Results by Machine ID:

# Count the number of times each unique value of Result occurs for each Supplier
result_by_supplier = pd.crosstab(dfb['Supplier'], dfb['Result'])

# Add a total column for reference
result_by_supplier['Total'] = result_by_supplier.sum(axis=1)

# Display the table
print("Count of Results by Supplier:")
result_by_supplier

Count of Results by Supplier:

# Group the data to count occurrences of each Result per Batch and machine_id
result_counts = dfb.groupby(['Batch', 'machine_id', 'Result']).size().reset_index(name='Count')

# Get unique machine IDs
machines = sorted(dfb['machine_id'].unique())

# Plot for each machine
for machine_id in machines:
    # Filter data for this machine
    machine_data = result_counts[result_counts['machine_id'] == machine_id]
    
    # Pivot data for stacking
    pivot_data = machine_data.pivot_table(
        index='Batch', columns='Result', values='Count', fill_value=0
    ).sort_index()
    
    # Ensure both result columns exist
    for result in [0, 1]:
        if result not in pivot_data.columns:
            pivot_data[result] = 0
    
    # Plot stacked bar chart
    pivot_data.plot(
        kind='bar', stacked=True, figsize=(12, 6),  # Smaller figure size for simplicity
        color=['#E34A33', '#3182BD'],              # Colors for Fail (0) and Pass (1)
        edgecolor='white', linewidth=0.7,          # Add edges to bars
        alpha=0.8                                  # Transparency
    )
    
    # Add title and labels
    plt.title(f'Count of Test Results by Batch for Machine {machine_id}', 
              fontsize=14, fontweight='bold')
    plt.xlabel('Batch', fontsize=12)
    plt.ylabel('Count', fontsize=12)
    
    # Add grid
    plt.grid(axis='y', linestyle='--', alpha=0.7)
    
    # Rotate x-axis labels for readability
    plt.xticks(rotation=45, ha='right')
    
    # Add legend
    plt.legend(title='Result', loc='upper right')
    
    # Adjust layout
    plt.tight_layout()
    
    # Show the plot
    plt.show()

# Group the data to count occurrences of each Result per Batch, machine_id, and Supplier
result_counts = dfb.groupby(['Batch', 'machine_id', 'Supplier', 'Result']).size().reset_index(name='Count')

# Set up the plotting style for better aesthetics
sns.set_style("whitegrid")

# Create a faceted bar plot
g = sns.catplot(
    data=result_counts,
    x='Batch', y='Count', hue='Result',  # x-axis: Batch, y-axis: Count, color by Result
    col='machine_id', row='Supplier',    # Facet by machine_id (columns) and Supplier (rows)
    kind='bar',                          # Use a bar plot
    height=4, aspect=1.5,               # Adjust the size of each subplot
    palette=['#E34A33', '#3182BD'],     # Colors for Fail (0) and Pass (1)
    edgecolor='black', linewidth=0.5    # Add edges to bars for clarity
)

# Adjust x-axis labels for better visibility
for ax in g.axes.flat:
    # Get the current x-tick positions and labels
    ticks = ax.get_xticks()
    labels = [str(int(label)) for label in result_counts['Batch'].unique()]  # Ensure labels are strings
    
    # Show every 5th label to avoid clutter
    new_labels = [label if i % 5 == 0 else '' for i, label in enumerate(labels)]
    
    # Set the ticks and labels correctly
    ax.set_xticks(ticks)  # Ensure ticks are set
    ax.set_xticklabels(new_labels[:len(ticks)], rotation=45, ha='right', fontsize=8)

# Add titles and adjust layout
g.set_titles("Machine {col_name}, Supplier {row_name}")
g.fig.suptitle("Count of Test Results by Batch", fontsize=16, fontweight='bold', y=1.05)

# Adjust layout to prevent overlap
plt.tight_layout()
g.fig.subplots_adjust(bottom=0.15)  # Add more space for rotated labels

# Show the plot
plt.show()

# Calculate the proportion of failures (where Result == 1 indicates a failure)
# Group by Batch and machine_id to get the total number of tests
total_tests = dfb.groupby(['Batch', 'machine_id']).size().reset_index(name='Total')

# Count the number of failures (Result == 1)
failures = dfb[dfb['Result'] == 1].groupby(['Batch', 'machine_id']).size().reset_index(name='Failures')

# Merge the two DataFrames
failure_prop = pd.merge(total_tests, failures, on=['Batch', 'machine_id'], how='left')

# Fill NaN values in Failures with 0 (if there were no failures)
failure_prop['Failures'] = failure_prop['Failures'].fillna(0)

# Calculate the proportion of failures
failure_prop['Failure_Rate'] = failure_prop['Failures'] / failure_prop['Total']

# Create a pivot table for the heatmap
failure_pivot = failure_prop.pivot(index='Batch', columns='machine_id', values='Failure_Rate')

# Create a heatmap to visualize the failure rates
plt.figure(figsize=(10, 12))  # Adjusted figure size for better readability
ax = sns.heatmap(
    failure_pivot, 
    annot=True, 
    cmap='RdYlGn_r',  # Red (high failure) to Green (low failure)
    fmt='.2f', 
    vmin=0, 
    vmax=1,
    cbar_kws={'label': 'Failure Rate'}
)
plt.title('Proportion of Test Failures by Batch and Machine ID', fontsize=16, fontweight='bold', pad=20)
plt.xlabel('Machine ID', fontsize=12, fontweight='bold')
plt.ylabel('Batch', fontsize=12, fontweight='bold')

# Add a note about the interpretation
plt.figtext(0.5, 0.01, "Note: According to problem hint, Result=1 represents FAILURE", 
           ha='center', fontsize=10, fontstyle='italic')
plt.tight_layout(rect=[0, 0.03, 1, 0.97])  # Make room for the note
plt.show()

# Create a line plot to show trends by machine
plt.figure(figsize=(15, 6))  # Adjusted figure size
for machine in sorted(dfb['machine_id'].unique()):
    machine_data = failure_prop[failure_prop['machine_id'] == machine]
    plt.plot(
        machine_data['Batch'], 
        machine_data['Failure_Rate'], 
        marker='o', 
        linewidth=2, 
        label=f'Machine {machine}',
        alpha=0.8
    )

plt.title('Failure Rate by Batch for Each Machine', fontsize=16, fontweight='bold', pad=20)
plt.xlabel('Batch', fontsize=12, fontweight='bold')
plt.ylabel('Failure Rate', fontsize=12, fontweight='bold')
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend(title='Machine ID', loc='upper right')
plt.ylim(0, 1)

# Improve x-axis readability
batches = failure_prop['Batch'].unique()
if len(batches) > 5:
    tick_positions = np.arange(0, len(batches), 5)  # Show every 5th batch
    tick_labels = [batches[i] if i < len(batches) else '' for i in tick_positions]
    plt.xticks(tick_positions, tick_labels, rotation=45, ha='right', fontsize=10)

plt.tight_layout()
plt.show()

# Calculate summary statistics by machine
machine_summary = failure_prop.groupby('machine_id').agg({
    'Failure_Rate': ['mean', 'min', 'max', 'std'],
    'Total': 'sum',  # Add total number of tests per machine
    'Failures': 'sum'  # Add total number of failures per machine
})
machine_summary.columns = ['Average Failure Rate', 'Min Failure Rate', 'Max Failure Rate', 'Std Dev', 'Total Tests', 'Total Failures']
machine_summary = machine_summary.round(3)

print("Failure Rate Summary by Machine:")
print(machine_summary)

# Identify batches with the highest failure rates
top_failure_batches = failure_prop.sort_values('Failure_Rate', ascending=False).head(10)
print("\nTop 10 Batch-Machine Combinations with Highest Failure Rates:")
print(top_failure_batches[['Batch', 'machine_id', 'Failures', 'Total', 'Failure_Rate']])

# Print clarification about the data interpretation
print("\nIMPORTANT CLARIFICATION:")
print("According to the problem hint, the data encoding is:")
print("- Result = 1: FAILED test")
print("- Result = 0: PASSED test")

Failure Rate Summary by Machine:
            Average Failure Rate  Min Failure Rate  Max Failure Rate  Std Dev  \
machine_id                                                                      
1                          0.314               0.0               1.0    0.309   
2                          0.300               0.0               1.0    0.326   
3                          0.271               0.0               1.0    0.313   

            Total Tests  Total Failures  
machine_id                               
1                   492           159.0  
2                   489           147.0  
3                   431           120.0  

Top 10 Batch-Machine Combinations with Highest Failure Rates:
     Batch  machine_id  Failures  Total  Failure_Rate
45      17           2       6.0      6           1.0
136     49           1       9.0      9           1.0
1        1           2      10.0     10           1.0
68      25           2      11.0     11           1.0
112     41           1      12.0     12           1.0
90      33           2      10.0     10           1.0
44      17           1       9.0      9           1.0
113     41           2      11.0     11           1.0
24       9           3      11.0     11           1.0
23       9           2       9.0      9           1.0

IMPORTANT CLARIFICATION:
According to the problem hint, the data encoding is:
- Result = 1: FAILED test
- Result = 0: PASSED test

# Calculate failure proportions by Batch, machine_id, and Supplier
# Remember that Result=1 represents failure according to problem hint
failure_prop_by_supplier = dfb.groupby(['Batch', 'machine_id', 'Supplier'])['Result'].mean().reset_index()
failure_prop_by_supplier.rename(columns={'Result': 'Failure_Rate'}, inplace=True)

# Get unique suppliers
suppliers = failure_prop_by_supplier['Supplier'].unique()
print(f"Unique suppliers: {suppliers}")

# Set up the figure size based on number of suppliers
fig, axes = plt.subplots(len(suppliers), 1, figsize=(16, len(suppliers)*8))

# If there's only one supplier, convert axes to array for consistent indexing
if len(suppliers) == 1:
    axes = [axes]

# Create a heatmap for each supplier
for i, supplier in enumerate(suppliers):
    # Filter data for this supplier
    supplier_data = failure_prop_by_supplier[failure_prop_by_supplier['Supplier'] == supplier]
    
    # Create a pivot table
    pivot_data = supplier_data.pivot(index='Batch', columns='machine_id', values='Failure_Rate')
    
    # Create the heatmap
    sns.heatmap(pivot_data, annot=True, cmap='RdYlGn_r', vmin=0, vmax=1, ax=axes[i], 
                cbar_kws={'label': 'Failure Rate'}, fmt='.2f')
    
    # Set titles and labels
    axes[i].set_title(f'Proportion of Test Failures - Supplier {supplier}', fontsize=16)
    axes[i].set_ylabel('Batch', fontsize=12)
    axes[i].set_xlabel('Machine ID', fontsize=12)

# Add a main title
plt.suptitle('Proportion of Test Failures by Batch, Machine, and Supplier', fontsize=18, y=1.02)

# Add a note about the data encoding
fig.text(0.5, -0.01, 'Note: Result=1 represents FAILURE according to problem hint', 
         ha='center', va='center', fontsize=12, style='italic')

# Adjust layout
plt.tight_layout()
plt.subplots_adjust(top=0.95, bottom=0.05)

# Display the figure
plt.show()

Unique suppliers: ['B' 'A']

# Extract the operational variables for clustering
X = dfb[['x1', 'x2', 'x3', 'x4']]

# Check for missing values
print("- Missing values in operational variables:")
print(X.isnull().sum())

# Standardize the variables for clustering
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_scaled_df = pd.DataFrame(X_scaled, columns=X.columns)

# Display summary statistics of the scaled data
print("\n- Standardized data summary:")
X_scaled_df.describe().round(2)

- Missing values in operational variables:
x1    0
x2    0
x3    0
x4    0
dtype: int64

- Standardized data summary:

wcss = []
for i in range(1, 11):
    kmeans = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=42)
    kmeans.fit(X_scaled)
    wcss.append(kmeans.inertia_)

# Plot the Elbow method results
plt.figure(figsize=(10, 6))
plt.plot(range(1, 11), wcss, marker='o', linestyle='-')
plt.title('Elbow Method for Optimal k', fontsize=16)
plt.xlabel('Number of clusters', fontsize=12)
plt.ylabel('WCSS', fontsize=12)
plt.grid(True, linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()

# Dendrogram for hierarchical clustering
# Calculate the linkage matrix
linkage_matrix = hierarchy.linkage(X_scaled, method='ward')

# Plot the dendrogram
plt.figure(figsize=(12, 8))
dendrogram = hierarchy.dendrogram(linkage_matrix, truncate_mode='level', p=5)
plt.title('Hierarchical Clustering Dendrogram', fontsize=16)
plt.xlabel('Sample index', fontsize=12)
plt.ylabel('Distance', fontsize=12)
plt.axhline(y=6, color='r', linestyle='--')  # Example threshold line
plt.tight_layout()
plt.show()

n_clusters = 3  

# Perform KMeans clustering
kmeans = KMeans(n_clusters=n_clusters, init='k-means++', max_iter=300, n_init=10, random_state=42)
dfb['cluster'] = kmeans.fit_predict(X_scaled)

# For hierarchical clustering, we'll use the linkage matrix from above
# and cut the tree to get n_clusters
hierarchical_clusters = hierarchy.fcluster(linkage_matrix, n_clusters, criterion='maxclust')
dfb['hierarchical_cluster'] = hierarchical_clusters - 1  # Adjust to 0-based indexing

# Compare the cluster assignments
print("Cluster distribution - KMeans:")
print(dfb['cluster'].value_counts().sort_index())
print("\nCluster distribution - Hierarchical:")
print(dfb['hierarchical_cluster'].value_counts().sort_index())

Cluster distribution - KMeans:
cluster
0    572
1    174
2    666
Name: count, dtype: int64

Cluster distribution - Hierarchical:
hierarchical_cluster
0    666
1    174
2    572
Name: count, dtype: int64

# Number of observations per cluster
fig, axes = plt.subplots(1, 2, figsize=(18, 6))

# KMeans cluster sizes
kmeans_counts = dfb['cluster'].value_counts().sort_index()
axes[0].bar(kmeans_counts.index, kmeans_counts.values, color='skyblue')
axes[0].set_title('Number of Observations per KMeans Cluster', fontsize=14)
axes[0].set_xlabel('Cluster', fontsize=12)
axes[0].set_ylabel('Count', fontsize=12)
axes[0].grid(axis='y', linestyle='--', alpha=0.7)

# Hierarchical cluster sizes
hierarchical_counts = dfb['hierarchical_cluster'].value_counts().sort_index()
axes[1].bar(hierarchical_counts.index, hierarchical_counts.values, color='lightgreen')
axes[1].set_title('Number of Observations per Hierarchical Cluster', fontsize=14)
axes[1].set_xlabel('Cluster', fontsize=12)
axes[1].set_ylabel('Count', fontsize=12)
axes[1].grid(axis='y', linestyle='--', alpha=0.7)

plt.tight_layout()
plt.show()

# Visualizing relationships between variables by cluster (using KMeans clusters)
cluster_col = 'cluster'

# Create a pairplot of the operational variables colored by cluster
sns.set_style('whitegrid')

plt.figure(figsize=(16, 12))
sns.pairplot(dfb, vars=['x1', 'x2', 'x3', 'x4'], hue=cluster_col, palette='viridis', 
             diag_kind='kde', plot_kws={'alpha': 0.6, 's': 80, 'edgecolor': 'k', 'linewidth': 0.5})
plt.suptitle('Relationships Between Operational Variables by Cluster', y=1.02, fontsize=16)
plt.tight_layout()
plt.show()

<Figure size 1600x1200 with 0 Axes>

# Calculate the proportion of failures in each cluster
failure_by_cluster = dfb.groupby(cluster_col)['Result'].mean().reset_index()
failure_by_cluster.rename(columns={'Result': 'Failure_Rate'}, inplace=True)

# Visualize failure rates by cluster
plt.figure(figsize=(10, 6))
bars = plt.bar(failure_by_cluster[cluster_col], failure_by_cluster['Failure_Rate'], 
               color=plt.cm.viridis(np.linspace(0, 1, len(failure_by_cluster))))

# Add value labels on top of bars
for bar in bars:
    height = bar.get_height()
    plt.text(bar.get_x() + bar.get_width()/2., height + 0.01,
            f'{height:.2f}', ha='center', va='bottom', fontsize=12)

plt.title('Proportion of Test Failures by Cluster', fontsize=16)
plt.xlabel('Cluster', fontsize=12)
plt.ylabel('Failure Rate', fontsize=12)
plt.ylim(0, max(failure_by_cluster['Failure_Rate']) * 1.1)  # Add 10% padding above highest bar
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()

# Create a detailed breakdown of cluster characteristics
cluster_stats = dfb.groupby(cluster_col).agg({
    'x1': ['mean', 'min', 'max', 'std'],
    'x2': ['mean', 'min', 'max', 'std'],
    'x3': ['mean', 'min', 'max', 'std'],
    'x4': ['mean', 'min', 'max', 'std'],
    'Result': 'mean'  # Failure rate
}).round(2)

print("Cluster Statistics and Failure Rates:")
cluster_stats

Cluster Statistics and Failure Rates:

# Apply PCA to reduce to 2 dimensions for visualization
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)

# Create a DataFrame with the PCA results and cluster labels
pca_df = pd.DataFrame(data=X_pca, columns=['PC1', 'PC2'])
pca_df['Cluster'] = dfb['cluster']
pca_df['Failure'] = dfb['Result']

# Plot the PCA results colored by cluster
plt.figure(figsize=(12, 8))
for cluster in pca_df['Cluster'].unique():
    cluster_data = pca_df[pca_df['Cluster'] == cluster]
    plt.scatter(cluster_data['PC1'], cluster_data['PC2'], 
                label=f'Cluster {cluster}', alpha=0.7, s=80)

plt.title('PCA of Clusters', fontsize=16)
plt.xlabel('Principal Component 1', fontsize=12)
plt.ylabel('Principal Component 2', fontsize=12)
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend()
plt.tight_layout()
plt.show()

# Plot the PCA results colored by failure status
plt.figure(figsize=(12, 8))
for result in [0, 1]:  # 0 = pass, 1 = fail
    result_data = pca_df[pca_df['Failure'] == result]
    plt.scatter(result_data['PC1'], result_data['PC2'], 
                label=f'{"Fail" if result==1 else "Pass"}', 
                alpha=0.7, s=80)

plt.title('PCA by Test Result', fontsize=16)
plt.xlabel('Principal Component 1', fontsize=12)
plt.ylabel('Principal Component 2', fontsize=12)
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend()
plt.tight_layout()
plt.show()

# Get statistics for each cluster to analyze their characteristics
cluster_stats = dfb.groupby('cluster').agg({
    'x1': ['mean', 'median', 'std'],
    'x2': ['mean', 'median', 'std'],
    'x3': ['mean', 'median', 'std'],
    'x4': ['mean', 'median', 'std'],
    'Result': 'mean'  # Failure rate
}).round(2)

# Identify the high-failure cluster
high_failure_cluster = failure_by_cluster.loc[failure_by_cluster['Failure_Rate'].idxmax(), 'cluster']
print(f"\nCluster {high_failure_cluster} has the highest failure rate at {failure_by_cluster['Failure_Rate'].max():.2f}")

print("\n- Cluster Statistics and Failure Rates:")
cluster_stats

Cluster 0 has the highest failure rate at 0.46

- Cluster Statistics and Failure Rates:

# Examine what batches are in the high-failure cluster
high_fail_batches = dfb[dfb['cluster'] == high_failure_cluster]['Batch'].value_counts().sort_values(ascending=False)
print(f"\nTop batches in high-failure Cluster {high_failure_cluster}:")
high_fail_batches

Top batches in high-failure Cluster 0:

Batch
12    37
36    36
2     36
33    36
41    35
34    35
42    33
44    31
9     30
20    30
49    30
26    26
1     25
17    25
4     24
28    23
10    21
25    20
18    20
50    19
Name: count, dtype: int64

	ID	Batch	s_id	x1	x2	x3	x4	machine_id
0	B001-M01-S001	1	1	50.117118	102.167346	22.067812	13.889524	1
1	B001-M01-S002	1	2	46.887333	112.266102	23.197330	13.064709	1
2	B001-M01-S003	1	3	50.132744	103.674908	22.319933	13.647482	1
3	B001-M01-S004	1	4	48.501042	107.143156	22.162947	14.077758	1
4	B001-M01-S005	1	5	49.690442	102.120283	22.248696	13.728666	1

	count	mean	std	min	25%	50%	75%	max
Supplier
A	30.0	9.793619	0.931674	8.703627	9.127575	9.460852	10.258287	12.246089
B	20.0	9.854414	0.363426	9.043240	9.658024	9.970531	10.085373	10.388587

	ID	Batch	s_id	x1	x2	x3	x4	machine_id	Supplier	Density
0	B001-M01-S001	1	1	50.117118	102.167346	22.067812	13.889524	1	B	10.388587
1	B001-M01-S002	1	2	46.887333	112.266102	23.197330	13.064709	1	B	10.388587
2	B001-M01-S003	1	3	50.132744	103.674908	22.319933	13.647482	1	B	10.388587
3	B001-M01-S004	1	4	48.501042	107.143156	22.162947	14.077758	1	B	10.388587
4	B001-M01-S005	1	5	49.690442	102.120283	22.248696	13.728666	1	B	10.388587

	ID	Batch	s_id	x1	x2	x3	x4	machine_id	Supplier	Density
0	B001-M01-S001	1	1	50.117118	102.167346	22.067812	13.889524	1	B	10.388587
1	B001-M01-S002	1	2	46.887333	112.266102	23.197330	13.064709	1	B	10.388587
2	B001-M01-S003	1	3	50.132744	103.674908	22.319933	13.647482	1	B	10.388587
3	B001-M01-S004	1	4	48.501042	107.143156	22.162947	14.077758	1	B	10.388587
4	B001-M01-S005	1	5	49.690442	102.120283	22.248696	13.728666	1	B	10.388587

	ID	test_group_id	Result
0	B001-M01-S056	A-0	1
1	B001-M01-S002	A-0	1
2	B001-M01-S049	A-0	1
3	B001-M01-S030	A-0	0
4	B001-M01-S048	A-0	1

	ID	test_group_id	Result	Batch	s_id	x1	x2	x3	x4	machine_id	Supplier	Density
0	B001-M01-S056	A-0	1	1	56	49.215485	105.647327	21.410531	14.154576	1	B	10.388587
1	B001-M01-S002	A-0	1	1	2	46.887333	112.266102	23.197330	13.064709	1	B	10.388587
2	B001-M01-S049	A-0	1	1	49	49.125629	112.323042	22.874947	13.743788	1	B	10.388587
3	B001-M01-S030	A-0	0	1	30	48.306819	109.445722	22.910858	13.839830	1	B	10.388587
4	B001-M01-S048	A-0	1	1	48	50.564504	109.184587	22.083064	13.954942	1	B	10.388587

	x1	x2	x3	x4
count	1412.00	1412.00	1412.00	1412.00
mean	-0.00	0.00	0.00	0.00
std	1.00	1.00	1.00	1.00
min	-2.21	-2.77	-2.07	-1.76
25%	-0.88	-0.46	-0.71	-0.73
50%	0.20	0.20	-0.03	-0.19
75%	0.90	0.67	0.61	0.44
max	2.15	1.85	2.28	2.57

	x1			x2			x3			x4			Result
	mean	median	std	mean	median	std	mean	median	std	mean	median	std	mean
cluster
0	50.37	49.78	1.78	98.93	99.66	5.72	21.83	22.16	1.02	11.82	11.10	1.44	0.46
1	46.85	46.86	0.69	75.07	74.97	2.90	26.61	26.63	0.37	9.39	9.39	0.22	0.40
2	54.59	54.82	1.38	111.20	110.57	5.41	25.05	24.42	1.59	9.86	9.68	1.02	0.14

Result	0	1	Total
test_group_id
A-0	1	6	7
A-1	8	4	12
A-2	5	4	9
A-3	9	2	11
A-4	11	3	14
A-5	9	1	10
B-0	10	3	13
B-1	7	0	7
B-2	8	0	8
B-3	9	0	9
B-4	10	0	10
B-5	8	3	11
C-0	8	2	10
C-1	8	5	13
C-2	9	3	12
C-3	8	0	8
C-4	0	11	11
C-5	1	11	12
D-0	4	2	6
D-1	8	1	9
D-2	10	0	10
D-3	9	1	10
D-4	9	5	14
D-5	11	1	12
E-0	10	0	10
E-1	7	0	7
E-2	7	2	9
E-3	0	10	10
E-4	11	0	11
E-5	8	5	13
F-0	10	0	10
F-1	7	3	10
F-2	11	0	11
F-3	7	4	11
F-4	9	0	9
F-5	7	3	10
G-0	12	0	12
G-1	0	14	14
G-2	0	9	9

Result	0	1	Total
machine_id
1	333	159	492
2	342	147	489
3	311	120	431

Result	0	1	Total
Supplier
A	588	290	878
B	398	136	534

CMPINF 2100: Midterm exam¶

Yevheniy Chuba¶

You are NOT allowed to collaborate with anyone on this exam.¶

Overview¶

Import modules¶

Problem 01¶

SOLUTION¶

1. Data Loading and Initial Preparation¶

2. ESSENTIAL Activity: Dataset Dimensions and Structure [Answers Printed Out]¶

3. MARGINAL Behavior: Operating Variable Distributions¶

4. RELATIONSHIPS Question: Batch Distribution Analysis¶

5. RELATIONSHIPS Question: Machine-level Operating Variable Analysis¶

6. RELATIONSHIPS Question: Batch-level Operating Variable Analysis¶

7. RELATIONSHIPS Question: Variable Interactions Across Machines¶

8. RELATIONSHIPS Question: Variable Interactions Across Batches¶

9. Data Integration: Concatenating Machine Datasets¶

Problem 02¶

SOLUTION¶

1. Data Loading and Initial Exploration [Answers Printed Out]¶

2. MARGINAL Behavior: Density Distribution Analysis¶

3. RELATIONSHIPS Question: Density by Supplier¶

4. RELATIONSHIPS Question: Average Density by Supplier¶

5. RELATIONSHIPS Question: Density-Batch Relationship by Supplier¶

6. Data Integration: Merging Supplier and Machine Data¶

7. RELATIONSHIPS Question: Operating Variables by Batch and Supplier¶

8. RELATIONSHIPS Question: Operating Variables Relationships by Supplier¶

Problem 03¶

SOLUTION¶

1. Data Loading and Initial Exploration¶

2. MARGINAL Behavior: Result Distribution Analysis¶

3. RELATIONSHIPS Question: Result Counts by Test Group¶

4. Data Integration: Merging Test and Production Data¶

Problem 04¶

SOLUTION¶

1. Count the number of times each unique value of Result occurs for each value of machine_id¶

2. Count the number of times each unique value of Result occurs for each value of Supplier¶

3. Visualize the number of times each unique value of Result occurs per Batch for each value of machine_id¶

4. Visualize the number of times each unique value of Result occurs per Batch for each value of machine_id and Supplier¶

5. Calculate the PROPORTION of times the cell phone case failed the test in each Batch per machine_id¶

6. Visualize the PROPORTION of times the cell phone case failed the test in each Batch per machine_id and for each unique value of Supplier¶

Problem 05¶

SOLUTION¶

1. Data Preparation for Clustering¶

2. Determining Optimal Number of Clusters: KMeans - Elbow Method¶

2. Determining Optimal Number of Clusters: Hierarchical Clustering - Dendrogram¶

3. Visualizing Cluster Results¶

4. Analyzing Failure Proportions by Cluster¶

5. Use PCA to visualize the clusters in 2D¶

6. Describing the high-failure cluster¶